﻿//FinamHistorySource.cs
//Copyright (c) 2013 StockSharp LLC, all rights reserved.
//This code module is part of StockSharp library.
//This code is licensed under the GNU GENERAL PUBLIC LICENSE Version 3.
//See the file License.txt for the license details.
//More info on: http://stocksharp.com

namespace StockSharp.Algo.History.Finam
{
	using System;
	using System.Collections.Generic;
	using System.Globalization;
	using System.IO;
	using System.Linq;
	using System.Net;
	using System.Runtime.Serialization;
	using System.Runtime.Serialization.Json;
	using System.Text;
	using System.Xml.Linq;

	using Ecng.Common;
	using Ecng.Collections;
	using Ecng.Interop;
	using Ecng.Net;

	using StockSharp.Algo.Candles;
	using StockSharp.Algo.Properties;
	using StockSharp.BusinessEntities;

	[DataContract]
	class MicexBoardBase
	{
		[DataMember(Name = "id")]
		public int Id { get; set; }

		[DataMember(Name = "title")]
		public string Title { get; set; }

		[DataMember(Name = "name")]
		public string Name { get; set; }
	}

	[DataContract]
	class MicexBoard : MicexBoardBase
	{
		[DataMember(Name = "board_group_id")]
		public int BoardId { get; set; }
	}

	//[DataContract]
	//class MicexSecurity
	//{
	//    [DataMember]
	//    public List<string[]> Metadata { get; set; }

	//    [DataMember]
	//    public Dictionary<string, object> Info { get; set; }
	//}

	/// <summary>
	/// Источник для скачивания исторических сделок с Finam.
	/// </summary>
	public class FinamHistorySource : BaseHistorySource
	{
		private sealed class MicexSecurityInfo
		{
			public string Board { get; set; }
			//public int Decimals { get; set; }
			public int MinLotSize { get; set; }
			public decimal MinStepSize { get; set; }
		}
		
		private const string _urlReferer = "http://www.finam.ru/analysis/export/default.asp";

		//private static readonly Uri _securityUrl = "http://www.finam.ru/scripts/export.js".To<Uri>();
		private static readonly Uri _securityUrl = "http://www.finam.ru/cache/icharts/icharts.js".To<Uri>();

		private readonly Dictionary<string, MicexSecurityInfo> _micexSecurities;

		/// <summary>
		/// Ключ, по которому в <see cref="Security.ExtensionInfo"/> будет присутствовать номер финамовского идентификатора рынка.
		/// </summary>
		public const string FinamMarketIdField = "FinamMarketId";

		/// <summary>
		/// Ключ, по которому в <see cref="Security.ExtensionInfo"/> будет присутствовать номер финамовского идентификатора инструмента.
		/// </summary>
		public const string FinamSecurityIdField = "FinamSecurityId";

		/// <summary>
		/// Создать <see cref="FinamHistorySource"/>.
		/// </summary>
		/// <param name="securityStorage">Интерфейс для доступа к хранилищу информации об инструментах.</param>
		public FinamHistorySource(ISecurityStorage securityStorage)
			: base(securityStorage)
		{
			var root = Resources.MicexSecurities.To<XDocument>().Root;

			if (root == null)
				throw new InvalidOperationException();

			_micexSecurities = CultureInfo.InvariantCulture.DoInCulture(() =>
				root.Elements().ToDictionary(
					elem => elem.GetAttributeValue("code"),
					elem => new MicexSecurityInfo
					{
						Board = elem.GetAttributeValue("board"),
						MinLotSize = elem.GetAttributeValue<int>("minLotSize"),
						//Decimals = GetAttribute(elem, "decimals").To<int>(),
						MinStepSize = elem.GetAttributeValue<decimal>("minStepSize"),
					}));
		}

		private static IEnumerable<T> Process<T>(string fileName, Func<string[]> download, Func<string[], IEnumerable<T>> converter)
		{
			if (fileName.IsEmpty())
				throw new ArgumentNullException("fileName");

			if (download == null)
				throw new ArgumentNullException("download");

			if (converter == null)
				throw new ArgumentNullException("converter");

			string[] lines;

			if (!File.Exists(fileName))
			{
				fileName.CreateDirIfNotExists();

				lines = download();
				File.WriteAllLines(fileName, lines);
			}
			else
				lines = File.ReadAllLines(fileName);

			try
			{
				return CultureInfo.InvariantCulture.DoInCulture(() => converter(lines).ToArray());
			}
			catch (Exception ex)
			{
			    File.Delete(fileName);
			    throw new InvalidOperationException("Файл '{0}' не был успешно распарсен и будет удален.".Put(fileName), ex);
			}
		}

		/// <summary>
		/// Получить исторические сделки.
		/// </summary>
		/// <param name="security">Инструмент, для которого необходимо получить все сделки. Если значение равно null, то возвращаются сделки по всем инструментам.</param>
		/// <param name="time">Время торговой сессии, для которой необходимо получить сделки.</param>
		/// <returns>Исторические сделки.</returns>
		public IEnumerable<Trade> GetTrades(Security security, DateTime time)
		{
			var marketId = GetFinamId(security);

			var dir = Path.Combine(DumpFolder, marketId + "_" + security.Code);
			var fullPath = Path.Combine(dir, "{0:0000}_{1:00}_{2:00}.txt".Put(time.Year, time.Month, time.Day));

			return Process(fullPath,
			() =>
			{
				var url = new FinamUrl()
					.Security(security)
					.From(time)
					.To(time)
					.TimeFrame(TimeSpan.Zero);

				return DownloadFinam(url.Uri);
			},
			lines =>
				from line in lines
					select line.Split(';')
				into parts
				select new Trade
				{
			       	Id = FixId(parts[4].To<long>()),
			       	Security = security,
					Price = parts[2].To<decimal>(),
					Volume = parts[3].To<decimal>().RemoveTrailingZeros(),
					Time = (parts[0] + parts[1]).ToDateTime("yyyyMMddHHmmss"),
				}
			);
		}

		// http://forum.finam.ru/index.php?showtopic=9941
		private static long FixId(long id)
		{
			if (id < 0)
				id = int.MaxValue + id;

			return id;
		}

		private void UpdateSecurityFromMicex(Security security)
		{
			var micexSecurity = _micexSecurities.TryGetValue(security.Code);
			if (micexSecurity != null)
			{
				security.Class = micexSecurity.Board;
				security.MinLotSize = micexSecurity.MinLotSize;
				//security.Decimals = micexSecurity.Decimals;
				security.MinStepSize = micexSecurity.MinStepSize;
			}
			else
			{
				var str =
					Encoding.UTF8.GetString(
						DownloadMicex("http://www.micex.ru/marketdata/find/board_groups_w_prefix?secid={0}".Put(security.Code).To<Uri>()));

				if (!str.Contains("Exception"))
				{
					var boardPreficies = (MicexBoardBase[]) new DataContractJsonSerializer(typeof (MicexBoardBase[]))
					                                        	.ReadObject(
					                                        		Encoding.UTF8.GetBytes(str.Replace("name:", "\"name\":")).To<Stream>());

					var boardInfo =
						DownloadMicex("http://www.micex.ru/marketdata/find/security_boards?secid={0}".Put(security.Code).To<Uri>());
					var boards =
						(MicexBoard[]) new DataContractJsonSerializer(typeof (MicexBoard[])).ReadObject(boardInfo.To<Stream>());

					if (boards.Length > 0)
					{
						var prefixId = boardPreficies.First().Id;

						var board = boards.FirstOrDefault(b => b.BoardId == prefixId);

						if (board != null)
						{
							security.Class = board.Name;

							var securityInfo =
								DownloadMicex(
									"http://www.micex.ru/issrpc/marketdata/stock/shares/daily/full/result.json?boardid={1}&secid={0}&lang=ru".Put(
										security.Code, security.Class).To<Uri>());

							//var micexSecurity = (MicexSecurity)new DataContractJsonSerializer(typeof(MicexSecurity)).ReadObject(securityInfo.To<Stream>());

							var secLine = Encoding.UTF8.GetString(securityInfo).Split('\n')[1].Replace("]", string.Empty);

							if (!secLine.IsEmpty())
							{
								var info = secLine
									.Replace("\"", "")
									.Replace("{", "")
									.Replace("}", "")
									.Split(", ")
									.Select(item => item.Split(':'))
									.ToDictionary(p => p[0], p => p[1]);

								security.MinLotSize = info["LOTSIZE"].To<int>();
								//security.Decimals = info["DECIMALS"].To<int>();

								// http://stocksharp.com/forum/yaf_postsm8573_Gidra-i-Finam.aspx#post8573
								// значение как типа 0.1, так и 1e-6. поэтому сначала double
								security.MinStepSize = (decimal)double.Parse(info["MINSTEP"], CultureInfo.InvariantCulture);
							}
						}
					}
				}
			}
		
			//security.Id += "@" + security.Class;

			security.ExchangeBoard = ExchangeBoard.Micex;
		}

		/// <summary>
		/// Загрузить информацию об инструментах.
		/// </summary>
		/// <returns>Информация об инструментах.</returns>
		public static FinamSecurityInfo[] DownloadSecurityInfo()
		{
			var lines = DownloadFinam(_securityUrl);

			Func<string, string, string[]> format = (str, by) =>
				str
					.Replace("var aEmitent{0}s = [".Put(by), string.Empty)
					.Replace("];", string.Empty)
					.Replace("'", string.Empty)
					.Split(',');

			var secIds = format(lines[0], "Id");
			var secNames = format(lines[1], "Name");
			var secCodes = format(lines[2], "Code");
			var secMarkets = format(lines[3], "Market");

			var decimals = lines[4]
					.Replace("var aEmitentDecp = {", string.Empty)
					.Replace("};", string.Empty)
					.Split(',')
					.Select(s => new KeyValuePair<string, int>(s.Split(':')[0], s.Split(':')[1].To<int>()))
					.Distinct()
					.ToDictionary();

			var fsi = new FinamSecurityInfo[secIds.Length];

			for (var i = 0; i < secIds.Length; i++)
			{
				var name = secNames[i];

				var start = name.IndexOf('(');
				var end = name.IndexOf(')');

				if (start != -1 && end > start)
				{
					name = name.Remove(start, end - start + 1).Trim();
				}

				fsi[i] = new FinamSecurityInfo
				{
					FinamSecurityId = secIds[i].To<long>(),
					FinamMarketId = secMarkets[i].To<long>(),
					Code = secCodes[i],
					Name = name,
					Decimals = decimals[secIds[i]]
				};
			}

			return fsi;
		}

		/// <summary>
		/// Загрузить свечи.
		/// </summary>
		/// <param name="security">Инструмент, для которого необходимо получить свечки.</param>
		/// <param name="beginDate">Дата начала данных.</param>
		/// <param name="endDate">Дата конца данных.</param>
		/// <param name="tf">Тайм-фрейм (1min, 5min, 10min, 15min, 30min, 1hour, 1day).</param>
		/// <returns>Загруженные свечки.</returns>
		public IEnumerable<Candle> GetCandles(Security security, DateTime beginDate, DateTime endDate, TimeSpan tf)
		{
			if (null == security)
				throw new ArgumentNullException("security");

			var marketId = GetFinamId(security);

			var dir = Path.Combine(DumpFolder, marketId + "_" + security.Code);
			var fullPath = Path.Combine(dir, "candles_{0}m_{1:0000}_{2:00}_{3:00}_{4:0000}_{5:00}_{6:00}.txt"
				.Put(tf.TotalMinutes, beginDate.Year, beginDate.Month, beginDate.Day, endDate.Year, endDate.Month, endDate.Day));

			return Process(fullPath,
			() =>
			{
				var url = new FinamUrl()
					.Security(security)
					.From(beginDate)
					.To(endDate)
					.TimeFrame(tf);

				return DownloadFinam(url.Uri);
			},
			lines =>
				from line in lines
				//DATE, TIME, OPEN, HIGH, LOW, CLOSE, VOL
				select line.Split(';')
					into parts
					let openTime = (parts[0] + parts[1]).ToDateTime("yyyyMMddHHmmss")
					select (Candle)new TimeFrameCandle
					{
						Security = security,
						OpenTime = openTime,
						CloseTime = openTime + tf,
						OpenPrice = parts[2].To<decimal>(),
						HighPrice = parts[3].To<decimal>(),
						LowPrice = parts[4].To<decimal>(),
						ClosePrice = parts[5].To<decimal>(),
						TotalVolume = parts[6].To<decimal>(),
						State = CandleStates.Finished,
						Arg = tf,
					}
			);
		}

		/// <summary>
		/// Получить новые инструменты.
		/// </summary>
		/// <returns>Новые инструменты.</returns>
		public IEnumerable<Security> GetNewSecurities()
		{
			var newSecurities = new List<Security>();

			var fsi = DownloadSecurityInfo();

			foreach (var t in fsi)
			{
				var securityId = t.FinamSecurityId;
				var marketId = t.FinamMarketId;
				var secCode = t.Code;
				var secName = t.Name;

				var security = SecurityStorage.LoadBy(FinamSecurityIdField, securityId);

				if (security == null)
				{
					security = new Security
					{
						Id = null,
						Code = secCode,
						Name = secName,
						Class = string.Empty,
						ExtensionInfo = new Dictionary<object, object>
						{
							{ FinamMarketIdField, marketId },
							{ FinamSecurityIdField, securityId },
						},
					};

					// <market name="ММВБ Акции" id="1">
					// <market name="ММВБ облигации" id="2">
					// <market name="ММВБ Внесписочные облигации" id="12">
					// <market name="ММВБ Архив" id="16">

					// <market name="РТС" id="3">
					// <market name="РТС-GAZ" id="10">
					// <market name="РТС-GTS" id="11">
					// <market name="Фьючерсы ФОРТС" id="14">
					// <market name="ФОРТС Архив" id="17">
					// <market name="RTS Standard" id="38">
					// <market name="RTS Board" id="20">
					// <market name="РТС Архив" id="18">
					// <market name="СПФБ" id="9">

					// <market name="Фьючерсы" id="7">
					// <market name="Мировые Индексы" id="6">
					// <market name="АДР" id="8">
					// <market name="Сырье" id="24">

					// <market name="Акции США" id="25">
					// <market name="US Bonds/Notes" id="26">
					// <market name="Отрасли экономики США" id="27">
					// <market name="ETF" id="28">
					// <market name="Индексы мировой экономики" id="30">
					switch (marketId)
					{
						case 1:
							security.ExchangeBoard = ExchangeBoard.Micex;
							UpdateSecurityFromMicex(security);
							break;
						case 3:
						case 10:
						case 11:
						case 38:
						case 20:
						case 18:
							security.ExchangeBoard = ExchangeBoard.Forts;
							break;
						case 14:
						case 17:
							security.Type = SecurityTypes.Future;
							security.ExchangeBoard = ExchangeBoard.Forts;
							break;
						default:
							security.ExchangeBoard = ExchangeBoard.Test;
							security.Class = "Finam";
							break;
					}

					security.Id = SecurityIdGenerator.GenerateId(security.Code, security.Class, security.ExchangeBoard);

					var s = SecurityStorage.LoadBy("Id", security.Id);

					// инструмент был создан не FinamHistorySource
					if (s != null)
					{
						security = s;

						security.ExtensionInfo[FinamSecurityIdField] = securityId;
						security.ExtensionInfo[FinamMarketIdField] = marketId;
					}
					else
						newSecurities.Add(security);

					SecurityStorage.Save(security);
				}
				else if (marketId != GetFinamId(security))// || security.Code != secCode)
				{
					security.ExtensionInfo[FinamMarketIdField] = marketId;
					//security.Code = secCode;

					SecurityStorage.Save(security);
				}
			}

			return newSecurities;
		}

		private static long GetFinamId(Security security)
		{
			if (security == null)
				throw new ArgumentNullException("security");

			if (security.ExtensionInfo == null)
				throw new ArgumentException("Инструмент {0} не содержит информацию об идентификаторе Финам.".Put(security));

			var marketId = security.ExtensionInfo.TryGetValue(FinamMarketIdField);

			if (marketId == null)
				throw new ArgumentException("Инструмент {0} не содержит информацию об идентификаторе Финам.".Put(security));

			if (!(marketId is long))
				throw new ArgumentException("Инструмент {0} содержит некорректный идентификатор Финама {1}.".Put(security, marketId));

			return (long)marketId;
		}

		private static string[] DownloadFinam(Uri url)
		{
			if (url == null)
				throw new ArgumentNullException("url");

			using (var client = new WebClientEx { Timeout = TimeSpan.FromMinutes(3), DecompressionMethods = DecompressionMethods.Deflate | DecompressionMethods.GZip })
			{
				client.Headers.Add(HttpRequestHeader.Referer, _urlReferer);
				client.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate,sdch");

				var result = client.DownloadString(url);
				return result.IsEmpty() ? new string[0] : result.Split(Environment.NewLine);
			}
		}

		private static byte[] DownloadMicex(Uri url)
		{
			using (var client = new WebClient())
			{
				client.Encoding = Encoding.UTF8;
				return client.DownloadData(url);
			}
		}
	}
}